package tk.mybatis.springboot.spider.processor;

import com.alibaba.fastjson.JSON;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import tk.mybatis.springboot.spider.entity.ErrorEntity;
import tk.mybatis.springboot.spider.entity.GkmCategory;
import tk.mybatis.springboot.spider.entity.ImgsEntity;
import tk.mybatis.springboot.spider.entity.ProductEntity;
import tk.mybatis.springboot.spider.utils.HttpClientUtils;
import tk.mybatis.springboot.spider.utils.UrlConfig;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.*;
import java.math.BigDecimal;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by xlp on 2017/12/19.
 */
public class GkmPageProcessor implements PageProcessor {
    private Site site = Site.me()
            .setRetryTimes(3) //这个重试一般用处不大
            .setTimeOut(30000)//在使用代理的情况下,这个需要设置,可以考虑调大线程数目
            .setSleepTime(5000)//使用代理了之后,代理会通过切换IP来防止反扒。同时,使用代理本身qps降低了,所以这个可以小一些
            .setCycleRetryTimes(3)//这个重试会换IP重试,是setRetryTimes的上一层的重试,不要怕三次重试解决一切问题。。
            .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36")
            .setUseGzip(true);

    @Override
    public void process(Page page) {
        /**
         * 分类列表
         */
        if (page.getUrl().regex(UrlConfig.GKM_CATEGORY).match()) {
            try {
                System.out.println("**********工控猫**********");
                String html = page.getHtml().get();
                Document doc = Jsoup.parse(html);
                Element products = doc.select("#modelQueryResult_div").first();
                String totalPageNumStr = products.select("#page").first().attr("data-page");
                System.out.println("商品总列表页数：" + totalPageNumStr);
                int totalPageNum = Integer.parseInt(totalPageNumStr);

                for (int i = 1; i <= totalPageNum; i++) {
                    // https://www.gongkongmall.com/product/select-model?categoryId=21&pageIndex=5#J_crumbsBar
                    String newUrl = page.getUrl().toString() + "&pageIndex=" + i + "#J_crumbsBar";
                    System.out.println("商品列表页分页：" + newUrl);
                    page.addTargetRequest(newUrl);
                }
            } catch (Exception e) {
                ErrorEntity errorEntity = new ErrorEntity();
                errorEntity.setErrorLink(page.getUrl().toString());
                errorEntity.setError(e.getCause());
                errorEntity.setErrorType("工控猫-分类别表");
                page.putField("errorLink", errorEntity);
                e.printStackTrace();
            }
        }
        /**
         * 商品列表页
         */
        if (page.getUrl().regex(UrlConfig.GKM_PAGE_INDEX).match()) {
            try {
                String html = page.getHtml().xpath("//*[@id=\"modelQueryResult_div\"]").get();
                Document document = Jsoup.parse(html);
                Element tbody = document.getElementsByTag("tbody").first();
                Elements trs = tbody.getElementsByTag("tr");
                for (Element tr : trs) {
                    String detailId = tr.attr("data-shopmodel").toString();
                    String newUrl = UrlConfig.GKM_DOMAIN + "product/detail/" + detailId;
                    System.out.println("商品详情页链接：" + newUrl);
                    page.addTargetRequest(newUrl);
                }
            } catch (Exception e) {
                ErrorEntity errorEntity = new ErrorEntity();
                errorEntity.setErrorLink(page.getUrl().toString());
                errorEntity.setError(e.getCause());
                errorEntity.setErrorType("工控猫-商品列表页");
                page.putField("errorLink", errorEntity);
                e.printStackTrace();

            }

        }
        /**
         * 商品详情
         */
        if (page.getUrl().regex(UrlConfig.GKM_DETAIL).match()) {
            try {
                String pid = "";
                ProductEntity productEntity = new ProductEntity();
                ImgsEntity imgsEntity = new ImgsEntity();
                String html = page.getHtml().get();
                Document doc = Jsoup.parse(html);
                Element catsE = doc.getElementsByClass("here").first();
                Elements cats = catsE.getElementsByTag("a");
                for (int i = 1; i < cats.size(); i++) {
                    String cat = cats.get(i).text();
                    System.out.println("第" + i + "级分类：" + cat);
                    switch (i) {
                        case 1:
                            productEntity.setProductCat1(cat);
                            break;
                        case 2:
                            productEntity.setProductCat2(cat);
                            break;
                        case 3:
                            productEntity.setProductCat3(cat);
                            break;
                        case 4:
                            productEntity.setProductCat4(cat);
                            break;
                        case 5:
                            productEntity.setProductCat5(cat);
                            break;
                        case 6:
                            productEntity.setProductCat6(cat);
                            break;
                        case 7:
                            productEntity.setProductCat7(cat);
                            break;
                        case 8:
                            productEntity.setProductCat8(cat);
                            break;
                    }
                }
                Element productImgE = doc.getElementsByClass("goods_pic fl").first();
                Element smallImagesE = productImgE.getElementsByClass("smallImg").first();
                Element productInfoE = doc.getElementsByClass("goods_info fl").first();
                String productName = productInfoE.getElementsByTag("h1").first().text();
                productEntity.setProductName(productName);
                System.out.println("==========商品名称==========");
                System.out.println(productEntity.getProductName());

                Element priceE = doc.getElementsByClass("detail_price_wrap").first();
                Element priceSalesE = priceE.getElementsByTag("b").first();
                Element priceMarketE = priceE.getElementsByTag("em").first();
                if (null != priceSalesE) {
                    String priceSales = priceSalesE.text();
                    productEntity.setPriceSales(new BigDecimal(priceSales));
                    System.out.println("==========售价==========");
                    System.out.println(productEntity.getPriceSales());
                }
                if (null != priceSalesE) {
                    String priceMarket = priceMarketE.text();
                    productEntity.setPriceMarket(new BigDecimal(priceMarket));
                    System.out.println("==========市场价==========");
                    System.out.println(productEntity.getPriceMarket());
                }

                Element child2 = productInfoE.child(2);
                Element no1E = child2.getElementsByTag("dd").first();
                if (null != no1E) {
                    String no1Str = no1E.text();
                    pid = no1Str;
                    productEntity.setProductNo(no1Str);
                    System.out.println("==========订货号==========");
                    System.out.println(productEntity.getProductNo());

                    Elements smallImgs = smallImagesE.getElementsByTag("img");
                    for (int i = 0; i < smallImgs.size(); i++) {
                        imgsEntity.setPid(pid);// 用订货号做唯一标示
                        String imgUrl = UrlConfig.GKM_DOMAIN + smallImgs.get(i).attr("src").toString();
                        System.out.println("==========商品图片==========");
                        imgsEntity.setProductImgPath(imgUrl);
                        imgsEntity.setType("1");
                        System.out.println(imgsEntity.getProductImgPath());
                        page.putField("productImgs", imgsEntity);
//                    downloadPicture(imgUrl, "../pic/" + productName.trim() + "_" + no1Str + "_" + i + ".jpg");
                    }
                }
                Element child3 = productInfoE.child(3);
                Element no2E = child3.getElementsByTag("dd").first();
                if (null != no2E) {
                    String no2Str = no2E.text();
                    productEntity.setProductSpec(no2Str);
                    System.out.println("==========型号==========");
                    System.out.println(productEntity.getProductSpec());
                }

                Element child4 = productInfoE.child(4);
                Element attrE = child4.getElementsByTag("dd").first();
                if (null != attrE) {
                    String attr = attrE.text();
                    productEntity.setProductAttribute(attr);
                    System.out.println("==========参数==========");
                    System.out.println(productEntity.getProductAttribute());
                }

                Element row1 = productInfoE.getElementsByClass("detail-row-one").first();
                Element brandE = row1.getElementsByTag("dd").first();
                if (null != brandE) {
                    String brand = brandE.text();
                    productEntity.setProductBrand(brand);
                    System.out.println("==========品牌==========");
                    System.out.println(productEntity.getProductBrand());
                }

                Element row2 = productInfoE.getElementsByClass("detail-row-two").first();
                Element setE = row2.getElementsByTag("dd").first();
                if (null != setE) {
                    String set = setE.text();
                    productEntity.setProductSet(set);
                    System.out.println("==========系列==========");
                    System.out.println(productEntity.getProductSet());
                }

                Element row3 = productInfoE.getElementsByClass("detail-row-three").first();
                Element catE = row3.getElementsByTag("dd").first();
                if (null != catE) {
                    String cat = catE.text();
                    productEntity.setProductCat(cat);
                    System.out.println("==========品类==========");
                    System.out.println(productEntity.getProductCat());
                }

                Element row4 = productInfoE.getElementsByClass("detail-row-four").first();
                Element unitE = row4.getElementsByTag("dd").first();
                if (null != unitE) {
                    String unit = unitE.text();
                    productEntity.setProductUnit(unit);
                    System.out.println("==========单位==========");
                    System.out.println(productEntity.getProductUnit());
                }

                Element detailE = doc.getElementsByClass("detail").first();
                Element detail1 = detailE.getElementsByClass("goods_details").first();
                Element detail2 = detailE.getElementsByClass("goods_norm").first();
                if (null != detail1) {
                    Elements detailImgE = detail1.getElementsByTag("img");
                    System.out.println("==========产品详情==========");
                    if (null != detailImgE) {
                        StringBuffer imgBuffer = new StringBuffer();
                        for (Element detailImg : detailImgE) {
                            String detailImgUrl = UrlConfig.GKM_DOMAIN + detailImg.attr("src").toString();
                            System.out.println("产品详情图片：" + detailImgUrl);
                            ImgsEntity detailImgEntity = new ImgsEntity();
                            detailImgEntity.setPid(pid);
                            detailImgEntity.setProductImgPath(detailImgUrl);
                            detailImgEntity.setType("2");
                            page.putField("detailImgs", detailImgEntity);

                        }
                    } else {
                        productEntity.setTabArea0("暂无");
                        System.out.println("没有产品详情图片！");
                    }
                }

                if (null != detail2) {
                    productEntity.setTabArea1(detail2.toString());
                    System.out.println("==========规格参数==========");
                    System.out.println(productEntity.getTabArea1());
                } else {
                    productEntity.setTabArea1("暂无");
                    System.out.println("没有规格参数！");
                }
                productEntity.setDataFrom("gkm");
                System.out.println("==========数据来源==========");
                System.out.println(productEntity.getDataFrom());
                page.putField("productEntity", productEntity);

            } catch (Exception e) {
                ErrorEntity errorEntity = new ErrorEntity();
                errorEntity.setErrorLink(page.getUrl().toString());
                errorEntity.setError(e.getCause());
                errorEntity.setErrorType("工控猫-商品详情");
                page.putField("errorLink", errorEntity);
                e.printStackTrace();
            }
        }
    }

    @Override
    public Site getSite() {
        return this.site;
    }

    /**
     * 从首页获取3级分类链接
     */
    public String[] getLevel3() {
        ArrayList<Integer> ids = new ArrayList<Integer>();

//      https://www.gongkongmall.com/product/public/category/list-on-sale-category-tree?_=1516070053567
        HttpClientUtils httpClientUtils = new HttpClientUtils(
                "https://www.gongkongmall.com/product/public/category/list-on-sale-category-tree?_=" + System.currentTimeMillis());
        String jsonResponse = httpClientUtils.getData();
        System.out.println(jsonResponse);
        try {
            GkmCategory gkmCategory = JSON.parseObject(jsonResponse, GkmCategory.class);
            List<GkmCategory.DataBean.ChildrenBeanXX> level = gkmCategory.getData().get(0).getChildren();
            System.out.println("一级分类：" + level.size());//9
            for (int i = 0; i < level.size(); i++) {
                System.out.println("******************");
                List<GkmCategory.DataBean.ChildrenBeanXX.ChildrenBeanX> level1 = level.get(i).getChildren();
                System.out.println(level.get(i).getName() + " 的分类：" + level1.size());// 15
                for (int j = 0; j < level1.size(); j++) {
                    GkmCategory.DataBean.ChildrenBeanXX.ChildrenBeanX childrenBeanX = level1.get(j);
                    int id = childrenBeanX.getId();
                    String name = childrenBeanX.getName();
                    System.out.println("二级分类 id：" + id);
                    System.out.println("二级分类 name：" + name);
                    System.out.println("三级分类个数：" + childrenBeanX.getChildren().size());
                    for (int k = 0; k < childrenBeanX.getChildren().size(); k++) {
                        GkmCategory.DataBean.ChildrenBeanXX.ChildrenBeanX.ChildrenBean childrenBeans = childrenBeanX.getChildren().get(k);
                        System.out.println("三级分类名称+id：" + childrenBeans.getName() + "-------" + childrenBeans.getId());
                        ids.add(childrenBeans.getId());
                    }
                    System.out.println("=======================================");
                }
            }

        } catch (Exception e) {
            System.out.println("解析json出错");
            e.printStackTrace();
        }
        String[] urls = new String[ids.size()];
        for (int i = 0; i < ids.size(); i++) {
            String url = "https://www.gongkongmall.com/product/select-model?categoryId=" + ids.get(i);
            System.out.println("三级分类链接：" + url);
            urls[i] = url;
        }

        return urls;
    }

    /**
     * 打包时需要注释掉main方法
     */
//    public static void main(String[] args) {
//
//        Spider.create(new GkmPageProcessor())
////                .addPipeline(new YPipeline())
////                .setDownloader(new DungProxyDownloader())
//                .thread(20)
//                .addUrl("https://www.gongkongmall.com/product/select-model?categoryId=75&pageIndex=242#J_crumbsBar")
//                .runAsync();
//    }

    /**
     * 链接url下载图片
     *
     * @param urlList
     * @param path
     */
    private static void downloadPicture(String urlList, String path) {
        URL url = null;
        try {
            url = new URL(urlList);
            DataInputStream dataInputStream = new DataInputStream(url.openStream());

            FileOutputStream fileOutputStream = new FileOutputStream(new File(path));
            ByteArrayOutputStream output = new ByteArrayOutputStream();

            byte[] buffer = new byte[1024];
            int length;

            while ((length = dataInputStream.read(buffer)) > 0) {
                output.write(buffer, 0, length);
            }
            fileOutputStream.write(output.toByteArray());
            dataInputStream.close();
            fileOutputStream.close();
        } catch (MalformedURLException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
